<?php

namespace StudyBuddy;

/**
 * Class for breaking up string into
 * tokens then appying stopwords filter
 * to result
 */
class Tokenizer {

    protected $aStopwords;

    public function __construct() {
        $stopwords = STUDYBUDDY_PATH . DIRECTORY_SEPARATOR . 'stopwords.txt';
        $this->aStopwords = explode(PHP_EOL, file_get_contents($stopwords));
    }

    /**
     * Break up string by tokens (words)
     * return array of tokens as long
     * as tokens are longer than 2 chars, not
     * a stopword and are alphanumeric
     *
     * @param string $string
     * @return array array of words
     */
    public function tokenize($string) {
        $a = explode(' ', $string);
        $ret = array();
        foreach ($a as $token) {
            $keyword = \mb_strtolower(trim($token));

            if (\mb_strlen($keyword) > 2 && !in_array($keyword, $this->aStopwords)) {
                $ret[] = $keyword;
            }
        }
        d('tokized: ' . print_r($ret, 1));

        return $ret;
    }

}
